package edu.hit.crawler.util;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.StringReader;
import java.net.URL;
import java.util.HashSet;
import java.util.Set;

import org.apache.xerces.xni.Augmentations;
import org.apache.xerces.xni.QName;
import org.apache.xerces.xni.XMLAttributes;
import org.apache.xerces.xni.XNIException;
import org.cyberneko.html.parsers.SAXParser;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

/**
 * A SAX parser that extract and correct all the outlinks in a HTML document
 * 
 * @author Song Liu (Lamfeeling@126.com)
 * 
 */
public class HtmlParser extends SAXParser{
	private Set<String> links = new HashSet<String>();
	private String context;

    public void startElement(QName tag, XMLAttributes attr, Augmentations arg)
    	    throws XNIException {
    	if (tag.rawname.equals("A") || tag.rawname.equals("AREA")) {
    	    String href = attr.getValue("href");
    	    URL u;
    	    try {
    		u = new URL(new URL(context), href);
    		String url = u.toString();
    		url = url.replaceAll("\\/\\.\\/", "\\/");
    		url = url.replaceAll("\\/[^\\/]+\\/\\.\\.\\/", "\\/");
    		// avoid redirect
    		// if (suffix.endsWith("index.htm")
    		// || suffix.endsWith("index.html")
    		// || suffix.endsWith("index.asp")
    		// || suffix.endsWith("default.aspx")
    		// || suffix.endsWith("index.php")
    		// || suffix.endsWith("index.jsp")) {
    		// url = url.substring(0, url.lastIndexOf("/") + 1);
    		// } else
    		if (u.getFile().trim().equals("") && !url.endsWith("/")) {
    		    // add the / to the end
    		    url = url + "/";
    		}
    		links.add(url);
    	    } catch (Exception e) {
    		// e.printStackTrace();
    	    }
    	}
    	// TODO Auto-generated method stub
    	super.startElement(tag, attr, arg);
        }
	public String[] extract(String context, String text) {
		this.context = context;
		links.clear();
		try {
			parse(new InputSource(new StringReader(text)));
		} catch (SAXException e) { 
			e.printStackTrace();
		} catch (IOException e) { //
			e.printStackTrace();
		}
		return links.toArray(new String[0]);
	}

	/**
	 * 使用Jsoup开源工具包，从原始网页中抽取正文内容
	 * 
	 * @param htmlText
	 * @return
	 */
	public static String extractPlainTextWithJsoup(String htmlText) {
		Document doc;
		String s = "";

		try {
			doc = org.jsoup.Jsoup.parse(htmlText);
			s = doc.text();
		} catch (Exception e) {
			System.out.println("Html parser error:" + e.getMessage());
		}

		return s;
	}

	public String[] j_extract(String context, String htmlText) {
		links.clear();
		Document doc;
		doc = org.jsoup.Jsoup.parse(htmlText, context);

		Elements resultLinks = doc.select("a[href]");
		Elements resultLinks2 = doc.select("area[href]");
		for (Element link : resultLinks) {
			String href = link.attr("abs:href");
			/*
			 * System.out.println("title: " + link.text());
			 * System.out.println("href: " + href);
			 */
			links.add(href);
		}
		for (Element link : resultLinks2) {
			String href = link.attr("abs:href");
			
			 // System.out.println("title: " + link.text());
			  //System.out.println("href: " + href);
			 
			links.add(href);
		}
		
		
		return links.toArray(new String[0]);
	}

	public static void main(String[] args) throws IOException {
		String str = "<A   href='index.htm'>kdsfj</a><A   href='index.htm'>kdsfj</a>"
				+ "<A   href='/index'>kdsfj</a>"
				+ "<A   href='indedx.htm'>k2dsfj</a>"
				+ "<A   href='indesx.htm'>kd2sfj</a>"
				+ "<A   href='indexa.htm'>kdsf444j</a>dfsd"
				+ "ewr"
				+ ""
				+ "few<A   href='inddex.htm'>kd4sfj</a>"
				+ "s"
				+ "<a href='http://pro.163.com/'>健康</a>";

		StringBuffer sb = new StringBuffer();
		BufferedReader bf = new BufferedReader(new FileReader("in"));
		String line;
		while ((line = bf.readLine()) != null) {
			sb.append(line);
		}

		HtmlParser p = new HtmlParser();
		// p.j_extract("http://www.baidu.com", str);
		for (String s : p.extract(
				"http://gouwu.hao123.com/",
				sb.toString())) {
			System.out.println(s);
		}
		/*
		 * for (String s : p.extract("http://www.csdn.net", sb.toString())) {
		 * System.out.println(s); }
		 */
	}

}
